from queue import Queue
import requests
import threading
import time
import os
import redis
import json
import sys
from index1 import upload
import re
import random
import platform

thread_count = 16
copies_count = thread_count * 4
requests.packages.urllib3.disable_warnings()
# 查看是linux还是win
sys_platform = platform.platform().lower()
sys_status = 2 if "windows" in sys_platform else 1
dir = 'G:/apply/' if sys_status == 2 else '/www/wwwroot/data/'
redis_db = redis.Redis(host='106.75.35.45', port=6379, db=5, password='953058009',
                       decode_responses=True, charset='UTF-8', encoding='UTF-8')


class DownloadThread(threading.Thread):
    def __init__(self, bytes_queue: Queue, url):
        super().__init__(daemon=True)
        self.bytes_queue = bytes_queue
        self.url = url

    def run(self):
        while not self.bytes_queue.empty():
            bytes_range = self.bytes_queue.get()
            headers = {
                "User-Agent": get_ua(),
                "Referer": 'https://www.xitongzu.com',
                "Range": "bytes={}".format(bytes_range[1])
            }
            try:
                response = requests.get(self.url, headers=headers, verify=False, timeout=30)
                with open("files/{}.tmp".format(bytes_range[0]), "wb") as f:
                    f.write(response.content)
            except Exception:
                return False


def get_file_size(url) -> int:
    try:
        headers = {
            "Referer": 'https://www.xitongzu.com',
            "User-Agent": get_ua()
        }
        response = requests.head(url, headers=headers, verify=False, timeout=30)
        file_length = int(response.headers['Content-Length'])
    except Exception:
        file_length = 0
    return file_length


def get_thread_download(file_length) -> list:
    bytes = Queue(copies_count)
    start_bytes = -1
    for i in range(copies_count):
        bytes_size = int(file_length / copies_count) * i
        if i == copies_count - 1: bytes_size = file_length
        bytes_length = "{}-{}".format(start_bytes + 1, bytes_size)
        bytes.put([i, bytes_length])
        start_bytes = bytes_size
    return bytes


def create_threading(bytes_queue, url):
    thread_list = []
    for i in range(thread_count):
        thread = DownloadThread(bytes_queue, url)
        thread.start()
        thread_list.append(thread)

    for thread in thread_list:
        thread.join()


def composite_file(filename):
    if os.path.isfile(filename): os.remove(filename)
    # 防止文件不存在
    try:
        with open(filename, "ab") as f:
            for i in range(copies_count):
                with open("files/{}.tmp".format(i), "rb") as bytes_f:
                    f.write(bytes_f.read())

        for i in os.listdir("files"):
            os.remove("files/{}".format(i))
    except Exception:
        return False


def hum_convert(value):
    units = ["B", "KB", "MB", "GB", "TB", "PB"]
    size = 1024.0
    for i in range(len(units)):
        if (value / size) < 1:
            return "%.2f%s" % (value, units[i])
        value = value / size


# 获取下载的文件数据
def get_redis_data(name=''):
    list = redis_db.get(name)
    return json.loads(list)


# 查看是否已下载过
def is_download(filename):
    title = 'down_27_win_log'
    status = 0
    if not redis_db.hexists(title, filename):
        redis_db.hset(title, filename, 0)
        status = 1
    return status


# 获取文件夹下所有子级的文件名称
def get_file():
    dirname = dir
    if not os.path.exists(dirname):
        print('该文件夹不存在')
        sys.exit()

    file_list = []
    # 递归获取子级
    for root, dirs, files in os.walk(dirname):
        for file in files:
            path = os.path.join(root, file)
            file_list.append(path)
    return file_list


# 验证豌豆荚
def checkwandoujia(download_url):
    url_list = [
        'wandoujia.com',
        'mobgslb.tbcache.com'
    ]
    ip = getOutterIP()
    info = redis_db.get('wandoujia' + ip)
    current_time = int(time.time())
    for url in url_list:
        if url in download_url:
            redis_db.set('wandoujia' + ip, current_time)
            if info and current_time - int(info) < 180:
                time.sleep(current_time - int(info))
                return True
    return True


def getOutterIP():
    ip = ''
    try:
        res = requests.get('https://myip.ipip.net', timeout=5).text
        ip = re.findall(r'(\d+\.\d+\.\d+\.\d+)', res)
        ip = ip[0] if ip else ''
    except:
        pass
    return ip


# 随机获得ua
def get_ua():
    first_num = random.randint(55, 62)
    third_num = random.randint(0, 3200)
    fourth_num = random.randint(0, 140)
    os_type = [
        '(Windows NT 6.1; WOW64)', '(Windows NT 10.0; WOW64)', '(X11; Linux x86_64)',
        '(Macintosh; Intel Mac OS X 10_12_6)'
    ]
    chrome_version = 'Chrome/{}.0.{}.{}'.format(first_num, third_num, fourth_num)

    ua = ' '.join(['Mozilla/5.0', random.choice(os_type), 'AppleWebKit/537.36',
                   '(KHTML, like Gecko)', chrome_version, 'Safari/537.36']
                  )
    return ua


# 去下载
def download(download_url, filename, num):
    startt = time.time()
    file_length = get_file_size(download_url)
    print('文件大小：' + str(hum_convert(file_length)))
    status = 0
    if file_length == 0:
        print(f"第{num}个文件->" + filename + "<-下载执行失败")
    else:
        copies_queue = get_thread_download(file_length)
        create_threading(copies_queue, download_url)
        composite_file(filename)
        # 检查大小是否一致
        size = os.path.getsize(filename)
        if size != file_length:
            os.remove(filename)
            print(f"第{num}个文件->{filename}<-下载大小不一致已删除")
            status = 2
        else:
            end = round(time.time() - startt, 2)
            print(f"第{num}个文件->{filename}<-下载用时" + str(end) + "秒")
            status = 1
    return status


# 不分片下载
def down(url, save_name):
    print('开始下载第二次')
    try:
        startt = time.time()
        filesize = get_file_size(url)
        headers = {
            "Referer": 'https://www.xitongzu.com',
            "User-Agnt": get_ua()
        }
        file = requests.get(url, allow_redirects=True, headers=headers, timeout=60)
        with open(save_name, 'wb') as f:
            f.write(file.content)
            f.close
        size = os.path.getsize(save_name)
        status = 1
        if filesize != size:
            os.remove(save_name)
            status = 2
            print('第二次下载大小不一致已删除')
        else:
            end = round(time.time() - startt, 2)
            print('第二次下载完成,用时' + str(end) + "秒")
    except:
        status = 2
        pass
    return status


# 去请求php分析
def analyes(filename, system, file_type, is_exist):
    if sys_status == 1:
        url = "http://analyse.km061.com/api/pool/analyes?filename=" + filename + "&system=" + str(
            system) + "&type=" + str(file_type) + "&is_exist=" + str(is_exist)
    else:
        url = "http://www.lexicon.com/api/pool/analyes?filename=" + filename + "&system=" + str(
            system) + "&type=" + str(file_type) + "&is_exist=" + str(is_exist)
    requests.packages.urllib3.disable_warnings()
    response = requests.get(url=url, verify=False)
    res = json.loads(response.text)
    print(res['msg'])
    return res


# 下载失败通知
def downerror(filename, system):
    if sys_status == 1:
        url = "http://analyse.km061.com/api/pool/downerror?filename=" + filename + "&status=" + str(
            sys_status) + "&system=" + str(system)
    else:
        url = "http://www.lexicon.com/api/pool/downerror?filename=" + filename + "&status=" + str(
            sys_status) + "&system=" + str(system)
    requests.packages.urllib3.disable_warnings()
    response = requests.get(url=url, verify=False)
    res = json.loads(response.text)
    print(res['msg'])
    return res


def main():
    # 开始下载
    rname = 'analyse_linux' if sys_status == 1 else 'analyse_win'
    redis_db.set(rname + '_log', 1)
    # 找出要下载的数据
    num = 0
    upload_num = 0
    for i in range(1, 100000):
        detail = redis_db.rpop(rname)
        if detail:
            num += 1
            print('')
            print(f"开始执行第{num}个下载")
            detail = json.loads(detail)
            for v in detail:
                # 如果无需下载
                if v['is_exist'] == 1:
                    print(f"该文件是oss数据无需下载")
                    analyes_res = analyes(v['filename'], v['system'], v['type'], v['is_exist'])
                    if analyes_res['code'] == 1:
                        print(f"已分析完成")
                    else:
                        print(f"分析出错")
                else:
                    filename = dir + v['filename']
                    system = '安卓' if v['system'] == 1 else 'pc'
                    print(f"开始执行{system}下载")
                    # 去下载
                    res = download(v['download_url'], filename, num)
                    # 去开启第二遍下载
                    if res == 2:
                        try:
                            res = down(v['download_url'], filename)
                            if res == 1:
                                file_size = os.path.getsize(filename)
                                if file_size < 800:
                                    os.remove(filename)
                                    res = 2
                        except:
                            res = 2
                    # 如果太小则删除
                    if res == 1:
                        # 去分析
                        analyes_res = analyes(filename, v['system'], v['type'], v['is_exist'])
                        if analyes_res['code'] == 1:
                            if int(v['is_create_url']) > 0:
                                # 去上传
                                upload_num += 1
                                upload(analyes_res['data']['path'], upload_num, v['type'])
                            else:
                                os.remove(analyes_res['data']['path'])
                        else:
                            # 失败去删除
                            os.remove(filename)

                    else:
                        # 通知失败
                        downerror(filename, v['system'])
                        # 如果是windows只要错一次就不继续往下执行
                        if sys_status == 2:
                            break
        else:
            redis_db.set(rname + '_log', 0)
            break


if __name__ == '__main__':
    main()
